# One substring per line (matched case-insensitively against User-Agent).
# Lines starting with # are ignored.

# Major search / social crawlers
googlebot
google-inspectiontool
adsbot-google
mediapartners-google
bingbot
msnbot
slurp
duckduckbot
baiduspider
yandexbot
yandeximages
yandexvideo
sogou
exabot
facebot
facebookexternalhit
ia_archiver
applebot
petalbot
bytespider

# AI / research crawlers
gptbot
chatgpt-user
claudebot
anthropic-ai
ccbot
perplexitybot
omgili
omgilibot
cohere-ai

# Link preview / messaging
twitterbot
linkedinbot
whatsapp
telegrambot
discordbot
pinterest
slackbot
skypeuripreview

# SEO / monitoring
ahrefsbot
semrushbot
mj12bot
dotbot
rogerbot
screaming frog
sitebulb
seokicks
sistrix

# Generic libraries and scripts
curl/
wget
python-requests
python-urllib
httpx/
aiohttp
libwww-perl
go-http-client
java/
scrapy
mechanize
phantomjs
headlesschrome
puppeteer
playwright
selenium

# Other crawlers / bots (keep patterns specific to avoid blocking real browsers)
crawler
spider
bot.htm
bot/
[bot]
scraper
fetcher
indexer
